# 导入文本分割器
from langchain.text_splitter import RecursiveCharacterTextSplitter, CharacterTextSplitter

if __name__ == '__main__':
    chunk_size = 20 #设置块大小
    chunk_overlap = 10 #设置块重叠大小
    # 初始化递归字符文本分割器
    r_splitter = RecursiveCharacterTextSplitter(
        chunk_size=chunk_size,
        chunk_overlap=chunk_overlap
    )
    # 初始化字符文本分割器
    c_splitter = CharacterTextSplitter(
        chunk_size=chunk_size,
        chunk_overlap=chunk_overlap
    )
    #测试文本
    text = "在AI的研究中，由于大模型规模非常大，模型参数很多，在大模型上跑完来验证参数好不好训练时间 成本很高，所以一般会在小模型上做消融实验来验证哪些改进是有效的再去大模型上做实验。"
    texts1 = r_splitter.split_text(text)
    print("texts1: ", texts1)

    # 字符文本分割器
    texts2 = c_splitter.split_text(text)
    print("texts2: ", texts2)

    # 设置,分隔符
    c_splitter = CharacterTextSplitter(
        chunk_size=chunk_size, chunk_overlap=chunk_overlap, separator='，'
    )
    texts3 = c_splitter.split_text(text)
    print("texts3: ", texts3)